package com.ruoyi.kafka.story;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.apache.commons.collections.CollectionUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;

/**
 * @program: biz-trade
 * @description: 小说爬取器
 * @author: zengchen
 * @create: 2020-07-29 15:19
 **/
public class StorySpider {

    //    private static String url = "http://www.biquge.info/22_22533/";
//    private static String url = "http://www.biquge.info/11_11451/";
//    private static String url = "http://www.biquge.info/74_74132/";
    private static String url = "http://www.jianlaixiaoshuo.com/jianlai";
    private static String url1 = "http://www.jianlaixiaoshuo.com";



    public static void main(String[] args) {
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        try {
            //屏蔽日志信息
//            LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
            //支持JavaScript
            webClient.getOptions().setJavaScriptEnabled(true);
            webClient.getOptions().setCssEnabled(false);
            webClient.getOptions().setActiveXNative(false);
            webClient.getOptions().setThrowExceptionOnScriptError(false);
            webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
            webClient.getOptions().setTimeout(2000);
            HtmlPage rootPage = webClient.getPage(url);
            //设置一个运行JavaScript的时间
            webClient.waitForBackgroundJavaScript(2000);
            String html = rootPage.asXml();
            Document doc = Jsoup.parse(html);
//            Element content2 = doc.getElementById("BookText");
            Element element = doc.getElementsByClass("chapterlist").first();
            Elements dds = element.getElementsByTag("dd");
            System.out.println("dds size=" + dds.size());

            List<DD> list = new ArrayList<>();
            for (int i = 48; i < dds.size(); i++) {
                Element story = dds.get(i);
                Elements as = story.getElementsByTag("a");
                if (CollectionUtils.isNotEmpty(as)) {
                    Element element1 = as.get(0);
                    String href = element1.attr("href");
                    String title = element1.wholeText().replaceAll("\r","").replaceAll("\n","").trim();
//                    System.out.println(i+"  "+title + "  " + href);
                    DD dd = new DD();
                    dd.setNum(i);
                    dd.setOrder(Integer.parseInt(href.replace("/book/","").replace(".html","")));
                    dd.setTitle(title);
                    dd.setUrl(href);
                    list.add(dd);
                }
            }

            List<DD> collect = list.stream().sorted(Comparator.comparing(DD::getOrder)).collect(Collectors.toList());
            for (int i = 3; i < collect.size(); i++) {
                DD dd = collect.get(i);
                System.out.println("顺序：" +i + "   序号：" + dd.getNum() + "  " + dd.getTitle());
                String detailUrl = url1 + dd.getUrl();
                HtmlPage detailPage = null;
                try {
                    detailPage = webClient.getPage(detailUrl);
                } catch (SocketTimeoutException e1) {
                    System.out.println("获取url内容出错,重新连接第一次");
                    Thread.sleep(1000);
                    try {
                        detailPage = webClient.getPage(detailUrl);
                    } catch (SocketTimeoutException e2) {
                        System.out.println("获取url内容出错,重新连接第二次");
                        Thread.sleep(1000);
                        detailPage = webClient.getPage(detailUrl);
                    }
                }
                String detailHtml = detailPage.asXml();
                Document detailDoc = Jsoup.parse(detailHtml);
                detailDoc.select("h4").remove();
                Element content = detailDoc.getElementById("BookText");
                Thread.sleep(1500);
                String text = content.wholeText();
                writeContent(dd.getTitle());
                writeContent(text);
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            webClient.close();
            System.out.println("webClient close 关闭连接");
        }
    }

    public static void writeContent(String txt) {
        FileWriter fw = null;
        try {
//            File f = new File("D:\\story\\修真界败类.txt");
            File f = new File("D:\\story\\剑来1.txt");
            fw = new FileWriter(f, true);
        } catch (IOException e) {
            e.printStackTrace();
        }
        PrintWriter pw = new PrintWriter(fw);
        pw.println(txt);
        pw.flush();
        try {
            fw.flush();
            pw.close();
            fw.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    static class DD {
        private int num;
        private int order;
        private String title;
        private String url;

        public int getNum() {
            return num;
        }

        public void setNum(int num) {
            this.num = num;
        }

        public int getOrder() {
            return order;
        }

        public void setOrder(int order) {
            this.order = order;
        }

        public String getTitle() {
            return title;
        }

        public void setTitle(String title) {
            this.title = title;
        }

        public String getUrl() {
            return url;
        }

        public void setUrl(String url) {
            this.url = url;
        }

        @Override
        public String toString() {
            return "DD{" +
                    "num=" + num +
                    ", order=" + order +
                    ", title='" + title + '\'' +
                    ", url='" + url + '\'' +
                    '}';
        }
    }
}
